{
    "cells": [
     {
      "cell_type": "code",
      "execution_count": 1,
      "id": "e5864f76-a9f0-49b7-9de9-93d49fd236ce",
      "metadata": {},
      "outputs": [
       {
        "data": {
         "text/plain": [
          "{'a',\n",
          " 'about',\n",
          " 'above',\n",
          " 'after',\n",
          " 'again',\n",
          " 'against',\n",
          " 'ain',\n",
          " 'all',\n",
          " 'am',\n",
          " 'an',\n",
          " 'and',\n",
          " 'any',\n",
          " 'are',\n",
          " 'aren',\n",
          " \"aren't\",\n",
          " 'as',\n",
          " 'at',\n",
          " 'be',\n",
          " 'because',\n",
          " 'been',\n",
          " 'before',\n",
          " 'being',\n",
          " 'below',\n",
          " 'between',\n",
          " 'both',\n",
          " 'but',\n",
          " 'by',\n",
          " 'can',\n",
          " 'couldn',\n",
          " \"couldn't\",\n",
          " 'd',\n",
          " 'did',\n",
          " 'didn',\n",
          " \"didn't\",\n",
          " 'do',\n",
          " 'does',\n",
          " 'doesn',\n",
          " \"doesn't\",\n",
          " 'doing',\n",
          " 'don',\n",
          " \"don't\",\n",
          " 'down',\n",
          " 'during',\n",
          " 'each',\n",
          " 'few',\n",
          " 'for',\n",
          " 'from',\n",
          " 'further',\n",
          " 'had',\n",
          " 'hadn',\n",
          " \"hadn't\",\n",
          " 'has',\n",
          " 'hasn',\n",
          " \"hasn't\",\n",
          " 'have',\n",
          " 'haven',\n",
          " \"haven't\",\n",
          " 'having',\n",
          " 'he',\n",
          " 'her',\n",
          " 'here',\n",
          " 'hers',\n",
          " 'herself',\n",
          " 'him',\n",
          " 'himself',\n",
          " 'his',\n",
          " 'how',\n",
          " 'i',\n",
          " 'if',\n",
          " 'in',\n",
          " 'into',\n",
          " 'is',\n",
          " 'isn',\n",
          " \"isn't\",\n",
          " 'it',\n",
          " \"it's\",\n",
          " 'its',\n",
          " 'itself',\n",
          " 'just',\n",
          " 'll',\n",
          " 'm',\n",
          " 'ma',\n",
          " 'me',\n",
          " 'mightn',\n",
          " \"mightn't\",\n",
          " 'more',\n",
          " 'most',\n",
          " 'mustn',\n",
          " \"mustn't\",\n",
          " 'my',\n",
          " 'myself',\n",
          " 'needn',\n",
          " \"needn't\",\n",
          " 'no',\n",
          " 'nor',\n",
          " 'not',\n",
          " 'now',\n",
          " 'o',\n",
          " 'of',\n",
          " 'off',\n",
          " 'on',\n",
          " 'once',\n",
          " 'only',\n",
          " 'or',\n",
          " 'other',\n",
          " 'our',\n",
          " 'ours',\n",
          " 'ourselves',\n",
          " 'out',\n",
          " 'over',\n",
          " 'own',\n",
          " 're',\n",
          " 's',\n",
          " 'same',\n",
          " 'shan',\n",
          " \"shan't\",\n",
          " 'she',\n",
          " \"she's\",\n",
          " 'should',\n",
          " \"should've\",\n",
          " 'shouldn',\n",
          " \"shouldn't\",\n",
          " 'so',\n",
          " 'some',\n",
          " 'such',\n",
          " 't',\n",
          " 'than',\n",
          " 'that',\n",
          " \"that'll\",\n",
          " 'the',\n",
          " 'their',\n",
          " 'theirs',\n",
          " 'them',\n",
          " 'themselves',\n",
          " 'then',\n",
          " 'there',\n",
          " 'these',\n",
          " 'they',\n",
          " 'this',\n",
          " 'those',\n",
          " 'through',\n",
          " 'to',\n",
          " 'too',\n",
          " 'under',\n",
          " 'until',\n",
          " 'up',\n",
          " 've',\n",
          " 'very',\n",
          " 'was',\n",
          " 'wasn',\n",
          " \"wasn't\",\n",
          " 'we',\n",
          " 'were',\n",
          " 'weren',\n",
          " \"weren't\",\n",
          " 'what',\n",
          " 'when',\n",
          " 'where',\n",
          " 'which',\n",
          " 'while',\n",
          " 'who',\n",
          " 'whom',\n",
          " 'why',\n",
          " 'will',\n",
          " 'with',\n",
          " 'won',\n",
          " \"won't\",\n",
          " 'wouldn',\n",
          " \"wouldn't\",\n",
          " 'y',\n",
          " 'you',\n",
          " \"you'd\",\n",
          " \"you'll\",\n",
          " \"you're\",\n",
          " \"you've\",\n",
          " 'your',\n",
          " 'yours',\n",
          " 'yourself',\n",
          " 'yourselves'}"
         ]
        },
        "execution_count": 1,
        "metadata": {},
        "output_type": "execute_result"
       }
      ],
      "source": [
       "'''get stopwords'''\n",
       "import pandas as pd\n",
       "#stopwords already saved from NLTK library in case of internet problem\n",
       "stopwords=pd.read_csv('stopwords.csv')['sw']\n",
       "swl=set(stopwords.tolist())\n",
       "swl"
      ]
     },
     {
      "cell_type": "code",
      "execution_count": 2,
      "id": "827f9b38-d51c-40fa-ae0c-3f4aeb3fd012",
      "metadata": {},
      "outputs": [
       {
        "data": {
         "text/plain": [
          "sentiment\n",
          "neutral       8638\n",
          "worry         8459\n",
          "happiness     5209\n",
          "sadness       5165\n",
          "love          3842\n",
          "surprise      2187\n",
          "fun           1776\n",
          "relief        1526\n",
          "hate          1323\n",
          "empty          827\n",
          "enthusiasm     759\n",
          "boredom        179\n",
          "anger          110\n",
          "Name: count, dtype: int64"
         ]
        },
        "execution_count": 2,
        "metadata": {},
        "output_type": "execute_result"
       }
      ],
      "source": [
       "'''unique names of sentiment and the counts of each'''\n",
       "import pandas as pd\n",
       "allSt=pd.read_csv('./tweet_emotions.csv')\n",
       "allSt[\"sentiment\"].value_counts()"
      ]
     },
     {
      "cell_type": "code",
      "execution_count": 3,
      "id": "1ca2f7cb-0d82-4b28-b31f-79340e059b16",
      "metadata": {},
      "outputs": [],
      "source": [
       "'''split'''\n",
       "def spl(allSt):\n",
       "    allSt['content']=allSt['content'].str.replace(r'\\@\\S+\\s','', regex=True).str.replace(r'[^A-Za-z\\s\\d]','', regex=True).str.lower().apply(lambda x: [word for word in x.split() if word not in swl])\n",
       "    pf=pd.DataFrame({'word':allSt['content'].explode()}).join(allSt.drop('content',axis=1))\n",
       "    dic={}\n",
       "    for i in allSt[\"sentiment\"].unique():\n",
       "        dic[i]=pf[pf['sentiment']==i]\n",
       "    ct={sen:df['word'].value_counts() for sen,df in dic.items()}\n",
       "    return ct\n",
       "ct=spl(allSt)"
      ]
     },
     {
      "cell_type": "code",
      "execution_count": 4,
      "id": "3bace69d-f22b-4bf3-b849-8ce804ef202c",
      "metadata": {},
      "outputs": [],
      "source": [
       "'''normalize (preparation for filtering out common words with no emotional inclinations)'''\n",
       "def nor(ct):\n",
       "    norCt={}\n",
       "    for sen,freqs in ct.items():\n",
       "        minFreq=freqs.min()\n",
       "        maxFreq=freqs.max()\n",
       "        norFreqs=((freqs-minFreq)/(maxFreq-minFreq))*100\n",
       "        #100/(maxFreq-minFreq) is the normalization ratio, because the normalization range is [0,100]\n",
       "        norCt[sen]=norFreqs\n",
       "    return norCt\n",
       "norCt=nor(ct)"
      ]
     },
     {
      "cell_type": "code",
      "execution_count": 5,
      "id": "877beee3-a9c1-46bb-b9ec-c6d64e896421",
      "metadata": {},
      "outputs": [
       {
        "name": "stdout",
        "output_type": "stream",
        "text": [
         "Removed: Index(['0', '09', '1', '10', '100', '1000', '1030', '10am', '10pm', '11',\n",
         "       ...\n",
         "       'yrs', 'yu', 'yucky', 'yum', 'yumm', 'yummy', 'yup', 'zealand', 'zone',\n",
         "       'zoo'],\n",
         "      dtype='object', name='word', length=3932)\n"
        ]
       }
      ],
      "source": [
       "'''filter out common words'''\n",
       "def com(norCt,ct):\n",
       "    combined=pd.concat(norCt.values()).apply(lambda x:(int(x) if type(x)==float else x))\n",
       "    ctDf=pd.concat(ct,axis=1)\n",
       "    stat=combined.groupby(combined.index).agg(['min','max','count',lambda x:x.mode().iloc[0] if not x.mode().empty else None]).rename(columns={'<lambda_0>':'mode'})\n",
       "    toRem=stat[(stat['count']>=5)&(((stat['max']-stat['min'])<=10)|((stat['max']-stat['mode'])<=10))].index\n",
       "    #if the frequency is not less than 5 and any value within (range,max-mode-distance) of its counts is not greater than 10, then filter this word out from all ctwords lists\n",
       "    ctDf=ctDf.drop(index=toRem,errors='ignore')\n",
       "    ct={sen:ctDf[sen].dropna().apply(lambda x:(int(x) if type(x)==float else x)).sort_values(ascending=False).head(10) for sen in ctDf.columns}\n",
       "    print('Removed:',toRem)\n",
       "    return ct\n",
       "result=com(norCt,ct)"
      ]
     },
     {
      "cell_type": "code",
      "execution_count": 6,
      "id": "59cc16d3-e11c-4188-87bc-4dd4a29bddf6",
      "metadata": {},
      "outputs": [
       {
        "data": {
         "text/plain": [
          "{'empty': word\n",
          " get        40\n",
          " bored      38\n",
          " go         37\n",
          " work       33\n",
          " day        32\n",
          " got        27\n",
          " home       27\n",
          " twitter    25\n",
          " cant       24\n",
          " going      24\n",
          " Name: empty, dtype: int64,\n",
          " 'sadness': word\n",
          " sad       373\n",
          " miss      339\n",
          " day       339\n",
          " work      324\n",
          " go        308\n",
          " get       301\n",
          " cant      275\n",
          " got       236\n",
          " really    217\n",
          " going     216\n",
          " Name: sadness, dtype: int64,\n",
          " 'enthusiasm': word\n",
          " good     55\n",
          " go       54\n",
          " get      45\n",
          " want     44\n",
          " day      39\n",
          " work     36\n",
          " new      34\n",
          " going    33\n",
          " cant     32\n",
          " u        30\n",
          " Name: enthusiasm, dtype: int64,\n",
          " 'neutral': word\n",
          " get      369\n",
          " go       366\n",
          " day      362\n",
          " good     353\n",
          " work     331\n",
          " going    314\n",
          " one      291\n",
          " back     288\n",
          " know     271\n",
          " got      266\n",
          " Name: neutral, dtype: int64,\n",
          " 'worry': word\n",
          " get       539\n",
          " cant      474\n",
          " go        447\n",
          " day       413\n",
          " got       389\n",
          " good      384\n",
          " work      383\n",
          " going     382\n",
          " know      355\n",
          " really    310\n",
          " Name: worry, dtype: int64,\n",
          " 'surprise': word\n",
          " day       123\n",
          " get       116\n",
          " oh        115\n",
          " got        99\n",
          " cant       98\n",
          " know       95\n",
          " good       93\n",
          " really     91\n",
          " see        87\n",
          " going      86\n",
          " Name: surprise, dtype: int64,\n",
          " 'love': word\n",
          " love       855\n",
          " day        762\n",
          " happy      592\n",
          " mothers    569\n",
          " good       317\n",
          " thanks     188\n",
          " u          180\n",
          " great      146\n",
          " lol        140\n",
          " really     135\n",
          " Name: love, dtype: int64,\n",
          " 'fun': word\n",
          " lol      177\n",
          " fun      141\n",
          " good     113\n",
          " haha     102\n",
          " day      102\n",
          " get       91\n",
          " go        88\n",
          " going     84\n",
          " u         80\n",
          " see       76\n",
          " Name: fun, dtype: int64,\n",
          " 'hate': word\n",
          " hate      209\n",
          " work       82\n",
          " get        77\n",
          " sucks      72\n",
          " cant       72\n",
          " really     63\n",
          " got        54\n",
          " go         53\n",
          " going      51\n",
          " want       51\n",
          " Name: hate, dtype: int64,\n",
          " 'happiness': word\n",
          " day       624\n",
          " good      555\n",
          " happy     408\n",
          " thanks    295\n",
          " great     292\n",
          " lol       276\n",
          " got       243\n",
          " fun       222\n",
          " haha      222\n",
          " get       221\n",
          " Name: happiness, dtype: int64,\n",
          " 'boredom': word\n",
          " bored      27\n",
          " work       15\n",
          " go         12\n",
          " really     12\n",
          " amp        10\n",
          " tired      10\n",
          " one         9\n",
          " getting     9\n",
          " boring      9\n",
          " cant        9\n",
          " Name: boredom, dtype: int64,\n",
          " 'relief': word\n",
          " day        146\n",
          " good       124\n",
          " thanks      90\n",
          " finally     84\n",
          " time        80\n",
          " back        78\n",
          " got         76\n",
          " home        72\n",
          " get         69\n",
          " work        69\n",
          " Name: relief, dtype: int64,\n",
          " 'anger': word\n",
          " get      8\n",
          " go       6\n",
          " going    5\n",
          " haha     5\n",
          " right    5\n",
          " know     5\n",
          " good     5\n",
          " got      5\n",
          " day      5\n",
          " work     5\n",
          " Name: anger, dtype: int64}"
         ]
        },
        "execution_count": 6,
        "metadata": {},
        "output_type": "execute_result"
       }
      ],
      "source": [
       "'execute'\n",
       "result"
      ]
     }
    ],
    "metadata": {
     "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
     },
     "language_info": {
      "codemirror_mode": {
       "name": "ipython",
       "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.4"
     }
    },
    "nbformat": 4,
    "nbformat_minor": 5
   }
   